@webcrawl/mcp 1.4.2 → 1.4.4

package/README.md

@@ -32,7 +32,7 @@ Add to `.cursor/mcp.json`, `~/.claude.json`, or your MCP client config:
 
 ### Optional: webcrawl.app cloud
 
-Local mode is the default and needs nothing else. To run the same tools on the hosted service, either:
+Local mode is the default and needs nothing else. The packaged server loads the local engine from `dist/core`; if that import fails it exits with an error instead of silently calling webcrawl.app. To run the same tools on the hosted service, either:
 
 - add `"WEBCRAWL_API_KEY": "wc_live_..."` to `env` above (get a key at [webcrawl.app](https://webcrawl.app)), or
 - connect your client to the hosted endpoint `https://webcrawl.app/mcp` and sign in in the browser. No local install is needed.

package/dist/bin.js

@@ -1,20 +1,18 @@
 #!/usr/bin/env node
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { WebcrawlClient, resolveConfig } from "@webcrawl/client";
+import { loadLocalEngineClient } from "./load-local-engine.js";
 import { createServer, resolveSchemaProfile } from "./server.js";
 const cfg = resolveConfig();
 let client;
 if (cfg.effectiveMode === "local") {
     try {
-        // Load the shared local engine when running from the monorepo / local install without an API key
-        const localEngineSpec = ["..", "..", "..", "src", "local-engine.js"].join("/");
-        const localMod = (await import(localEngineSpec));
-        client = localMod.createLocalEngineClient();
+        client = await loadLocalEngineClient();
         process.stderr.write("webcrawl-mcp: running in zero-key LOCAL engine mode. Set WEBCRAWL_API_KEY (or run `webcrawl login`) to upgrade to webcrawl.app cloud.\n");
     }
-    catch {
-        process.stderr.write("webcrawl-mcp: no API key configured; falling back to https://webcrawl.app (set WEBCRAWL_API_KEY or run `webcrawl login`).\n");
-        client = new WebcrawlClient({ apiKey: cfg.apiKey, baseUrl: cfg.baseUrl, userAgent: "webcrawl-mcp/0.1" });
+    catch (e) {
+        process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
+        process.exit(1);
     }
 }
 else {

package/dist/core/engine/awswaf.d.ts

@@ -0,0 +1,59 @@
+/**
+ * AWS WAF challenge handling (local prototype).
+ *
+ * `webcrawl`'s bot-check detector (`isAntiBotChallenge`) only matches
+ * Cloudflare and Vercel Security Checkpoint signatures. AWS WAF serves
+ * its invisible JS challenge as **HTTP 202** with an empty `<title>` and
+ * none of those markers, so such pages come back as
+ * `Empty page (HTTP 202)` with `blocked: false` and never reach the
+ * stealth browser. This module adds the missing detection plus a local
+ * stealth-routing path so those pages can be solved and read anyway.
+ *
+ * Detection markers are layered (no single one is proof):
+ *   - status 202 (documented Challenge interstitial status)
+ *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
+ *   - `aws-waf-token` cookie reference
+ *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
+ */
+import { isAwsWafChallenge } from "./fetcher.js";
+export { isAwsWafChallenge };
+/** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
+export declare function isBlockedLike(html: string, status?: number): boolean;
+export interface AwsWafPage {
+    url: string;
+    title: string;
+    markdown: string;
+    linksCount: number;
+    cached: false;
+    cacheLayer: "miss";
+    structuredData: null;
+    statusCode?: number;
+    blocked?: boolean;
+    error?: string;
+    awsWaf?: boolean;
+    stealth?: boolean;
+}
+/**
+ * Opens `url` in a real local Chromium, waits out the AWS WAF challenge
+ * (the interstitial reloads itself once `AwsWafIntegration.getToken()`
+ * resolves), then returns the rendered page as a CrawledPage-shaped result.
+ * Never fabricates content: a page still showing the challenge comes back
+ * `blocked: true` with an honest error.
+ */
+export declare function awsWafStealthFetch(url: string, opts?: {
+    timeoutMs?: number;
+}): Promise<AwsWafPage>;
+/**
+ * Reads one page: plain TLS-impersonated fetch first; when the site answers
+ * with an AWS WAF challenge (which the upstream detector misses), routes
+ * through the local stealth browser instead.
+ */
+export declare function crawlPageWithAwsWaf(url: string, opts?: {
+    bypassCache?: boolean;
+    stealthTimeoutMs?: number;
+}): Promise<AwsWafPage>;

package/dist/core/engine/awswaf.js

@@ -0,0 +1,164 @@
+/**
+ * AWS WAF challenge handling (local prototype).
+ *
+ * `webcrawl`'s bot-check detector (`isAntiBotChallenge`) only matches
+ * Cloudflare and Vercel Security Checkpoint signatures. AWS WAF serves
+ * its invisible JS challenge as **HTTP 202** with an empty `<title>` and
+ * none of those markers, so such pages come back as
+ * `Empty page (HTTP 202)` with `blocked: false` and never reach the
+ * stealth browser. This module adds the missing detection plus a local
+ * stealth-routing path so those pages can be solved and read anyway.
+ *
+ * Detection markers are layered (no single one is proof):
+ *   - status 202 (documented Challenge interstitial status)
+ *   - `awsWafCookieDomainList` / `AwsWafIntegration` (documented JS API)
+ *   - `aws-waf-token` cookie reference
+ *   - `<div id="challenge-container">` + `/__challenge_<id>/.../challenge.js`
+ *
+ * The detection itself now lives in the shared detector
+ * (`isAwsWafChallenge` in `./fetcher.js`, also folded into
+ * `isAntiBotChallenge`); this module re-exports it and adds the local
+ * stealth-routing path so challenge pages can be solved and read anyway.
+ */
+import { fetchWithTlsImpersonation, isAntiBotChallenge, isAwsWafChallenge } from "./fetcher.js";
+import { fastHtmlToMarkdown } from "./markdown.js";
+export { isAwsWafChallenge };
+/** True when a fetch response is any known bot check (upstream detector + AWS WAF). */
+export function isBlockedLike(html, status) {
+    return isAntiBotChallenge(html ?? "", status ?? 200) || isAwsWafChallenge(html ?? "", status);
+}
+const MAC_CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
+const CHALLENGE_TIMEOUT_MS = 45_000;
+/** Waits for a dynamically-installed playwright-core (avoids a hard dep). */
+async function loadPlaywrightCore() {
+    // Non-literal specifier: playwright-core stays a soft, runtime-only dependency
+    // so `tsc --noEmit` passes without the package installed.
+    const specifier = "playwright-core";
+    return (await import(specifier));
+}
+function chromePath() {
+    return process.env.INTERACT_CHROME_PATH || MAC_CHROME;
+}
+/**
+ * Opens `url` in a real local Chromium, waits out the AWS WAF challenge
+ * (the interstitial reloads itself once `AwsWafIntegration.getToken()`
+ * resolves), then returns the rendered page as a CrawledPage-shaped result.
+ * Never fabricates content: a page still showing the challenge comes back
+ * `blocked: true` with an honest error.
+ */
+export async function awsWafStealthFetch(url, opts = {}) {
+    const fail = (error, blocked = true) => ({
+        url,
+        title: url,
+        markdown: "",
+        linksCount: 0,
+        cached: false,
+        cacheLayer: "miss",
+        structuredData: null,
+        blocked,
+        error,
+        awsWaf: true,
+        stealth: true,
+    });
+    const { chromium } = await loadPlaywrightCore();
+    const executablePath = chromePath();
+    const browser = await chromium.launch({
+        executablePath,
+        headless: true,
+        args: [
+            "--no-sandbox",
+            "--disable-gpu",
+            "--disable-dev-shm-usage",
+            "--no-first-run",
+            "--no-default-browser-check",
+            // Hide navigator.webdriver, which bot checks look for.
+            "--disable-blink-features=AutomationControlled",
+            `--window-size=1280,800`,
+        ],
+    });
+    try {
+        const ctx = await browser.newContext({
+            userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+            viewport: { width: 1280, height: 800 },
+            locale: "en-US",
+        });
+        await ctx.addInitScript("Object.defineProperty(Navigator.prototype,'webdriver',{get:()=>false});");
+        const page = await ctx.newPage();
+        let navStatus = null;
+        page.on("response", (r) => {
+            try {
+                if (r.request().isNavigationRequest() && r.frame() === page.mainFrame())
+                    navStatus = r.status();
+            }
+            catch { }
+        });
+        // The interstitial reloads itself after getToken(); domcontentloaded +
+        // marker polling survives that reload race.
+        await page.goto(url, { waitUntil: "domcontentloaded", timeout: Math.min(30_000, opts.timeoutMs ?? 30_000) });
+        const deadline = Date.now() + (opts.timeoutMs ?? CHALLENGE_TIMEOUT_MS);
+        let html = "";
+        for (;;) {
+            html = await page.content().catch(() => "");
+            if (!isAwsWafChallenge(html) && !isAntiBotChallenge(html, 200))
+                break;
+            if (Date.now() > deadline)
+                break;
+            await page.waitForTimeout(700).catch(() => { });
+        }
+        try {
+            await page.waitForLoadState("networkidle", { timeout: 2500 });
+        }
+        catch { }
+        html = await page.content().catch(() => html);
+        if (isAwsWafChallenge(html) || isAntiBotChallenge(html, 200)) {
+            return { ...fail("Still blocked by the site's AWS WAF challenge in local stealth mode"), statusCode: navStatus ?? undefined };
+        }
+        const finalUrl = page.url();
+        const parsed = fastHtmlToMarkdown(html, finalUrl || url, { retainImages: "all" });
+        if (parsed.markdown.length === 0) {
+            return {
+                ...fail(`Empty page after challenge (HTTP ${navStatus ?? "?"})`, false),
+                statusCode: navStatus ?? undefined,
+            };
+        }
+        return {
+            url,
+            title: parsed.title || finalUrl || url,
+            markdown: parsed.markdown,
+            linksCount: parsed.links.length,
+            cached: false,
+            cacheLayer: "miss",
+            structuredData: null,
+            statusCode: navStatus ?? 200,
+            awsWaf: true,
+            stealth: true,
+        };
+    }
+    finally {
+        await browser.close().catch(() => { });
+    }
+}
+/**
+ * Reads one page: plain TLS-impersonated fetch first; when the site answers
+ * with an AWS WAF challenge (which the upstream detector misses), routes
+ * through the local stealth browser instead.
+ */
+export async function crawlPageWithAwsWaf(url, opts = {}) {
+    const fetched = await fetchWithTlsImpersonation(url, 12_000);
+    if (!isAwsWafChallenge(fetched.html, fetched.status)) {
+        const parsed = fastHtmlToMarkdown(fetched.html, url, { retainImages: "all" });
+        return {
+            url,
+            title: parsed.title || url,
+            markdown: parsed.markdown,
+            linksCount: parsed.links.length,
+            cached: false,
+            cacheLayer: "miss",
+            structuredData: null,
+            statusCode: fetched.status,
+            ...(parsed.markdown.length === 0 ? { error: `Empty page (HTTP ${fetched.status})` } : {}),
+        };
+    }
+    console.info("[awswaf] challenge detected, routing through local stealth browser", { url, status: fetched.status });
+    return awsWafStealthFetch(url, { timeoutMs: opts.stealthTimeoutMs });
+}

package/dist/core/engine/cache.d.ts

@@ -0,0 +1,70 @@
+interface ExternalCacheDb {
+    getCrawledPage(urlHash: string): Promise<{
+        title: string;
+        markdown: string;
+        links: string[];
+        structuredData?: Record<string, unknown> | null;
+        statusCode: number;
+        contentHash: string;
+        etag?: string;
+        lastModified?: string;
+        crawledAt: string | Date;
+        expiresAt: string | Date;
+    } | null | undefined>;
+    upsertCrawledPage(entry: Record<string, unknown>): Promise<unknown>;
+    getCrawledSite(cacheKeyHash: string): Promise<{
+        links: string[];
+        source: string;
+        pagesCrawled: number;
+        crawledAt: string | Date;
+        expiresAt: string | Date;
+    } | null | undefined>;
+    upsertCrawledSite(entry: Record<string, unknown>): Promise<unknown>;
+}
+/**
+ * Registers an external L3 cache database store (e.g., Supabase Postgres in webcrawl.app cloud)
+ * when the engine is consumed as an installed npm package (`webcrawl/engine/cache`).
+ */
+export declare function registerExternalCacheDb(db: ExternalCacheDb | null): void;
+export interface CacheEntry {
+    title: string;
+    markdown: string;
+    links: string[];
+    structuredData?: Record<string, unknown> | null;
+    statusCode: number;
+    contentHash: string;
+    etag?: string;
+    lastModified?: string;
+    cacheLayer?: "memory" | "disk" | "postgres";
+    cachedAt: number;
+    expiresAt: number;
+}
+export interface MapCacheEntry {
+    links: string[];
+    source: string;
+    pagesCrawled: number;
+    cacheLayer?: "memory" | "disk" | "postgres";
+    cachedAt: number;
+    expiresAt: number;
+}
+export declare const CACHE_TTL_MS: number;
+export declare function sha256(content: string): string;
+export declare function normalizeCacheUrl(rawUrl: string): {
+    normalizedUrl: string;
+    domain: string;
+};
+export declare function clearLocalCacheForTests(): void;
+export declare function getCachedPage(url: string, retainImages: string): CacheEntry | undefined;
+/**
+ * 3-tier read-through lookup: L1 Memory -> L2 /tmp Disk -> L3 Postgres `crawled_pages`.
+ * Promotes L3 Postgres hits back into L1 and L2 so repeat requests never re-crawl the target website.
+ */
+export declare function getCachedPageAsync(url: string, retainImages: string): Promise<CacheEntry | undefined>;
+export declare function setCachedPage(url: string, retainImages: string, entry: Omit<CacheEntry, "expiresAt" | "cachedAt">, ttlMs?: number): void;
+export declare function getCachedMap(cacheKey: string): MapCacheEntry | undefined;
+/**
+ * 3-tier read-through lookup for website maps/crawls: L1 Memory -> L2 Disk -> L3 Postgres `crawled_sites`.
+ */
+export declare function getCachedMapAsync(cacheKey: string): Promise<MapCacheEntry | undefined>;
+export declare function setCachedMap(cacheKey: string, entry: Omit<MapCacheEntry, "cachedAt" | "expiresAt">, ttlMs?: number, rootUrl?: string): void;
+export {};

package/dist/core/engine/cache.js

@@ -0,0 +1,297 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+let cachedDbPromise = null;
+/**
+ * Registers an external L3 cache database store (e.g., Supabase Postgres in webcrawl.app cloud)
+ * when the engine is consumed as an installed npm package (`webcrawl/engine/cache`).
+ */
+export function registerExternalCacheDb(db) {
+    cachedDbPromise = Promise.resolve(db);
+}
+function getOptionalDb() {
+    if (!cachedDbPromise) {
+        const dbModulePath = "../db/index.js";
+        cachedDbPromise = import(/* @vite-ignore */ dbModulePath)
+            .then((mod) => mod?.db ?? null)
+            .catch(() => null);
+    }
+    return cachedDbPromise;
+}
+// 24-hour cache (L1 in-memory + L2 /tmp disk + L3 Postgres shared URL/website store)
+export const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
+const MAX_CACHE_ENTRIES = 2500;
+const TMP_CACHE_DIR = "/tmp/webcrawl-cache-v1";
+const pageCache = new Map();
+const mapCache = new Map();
+try {
+    fs.mkdirSync(TMP_CACHE_DIR, { recursive: true });
+}
+catch {
+    // ignore if read-only filesystem
+}
+export function sha256(content) {
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+export function normalizeCacheUrl(rawUrl) {
+    try {
+        const u = new URL(rawUrl.trim());
+        u.hash = "";
+        for (const key of Array.from(u.searchParams.keys())) {
+            if (/^(utm_|fbclid|gclid|msclkid|_ga|_gl|ref$)/i.test(key)) {
+                u.searchParams.delete(key);
+            }
+        }
+        const normalizedUrl = u.toString().replace(/\/$/, "") || u.origin;
+        const domain = u.hostname.toLowerCase().replace(/^www\./, "");
+        return { normalizedUrl, domain };
+    }
+    catch {
+        return {
+            normalizedUrl: rawUrl.trim(),
+            domain: "unknown",
+        };
+    }
+}
+export function clearLocalCacheForTests() {
+    pageCache.clear();
+    mapCache.clear();
+    try {
+        if (fs.existsSync(TMP_CACHE_DIR)) {
+            for (const f of fs.readdirSync(TMP_CACHE_DIR)) {
+                try {
+                    fs.unlinkSync(path.join(TMP_CACHE_DIR, f));
+                }
+                catch {
+                    // ignore
+                }
+            }
+        }
+    }
+    catch {
+        // ignore
+    }
+}
+export function getCachedPage(url, retainImages) {
+    const { normalizedUrl } = normalizeCacheUrl(url);
+    const key = `${normalizedUrl}::${retainImages}`;
+    const entry = pageCache.get(key);
+    if (entry) {
+        if (Date.now() > entry.expiresAt) {
+            pageCache.delete(key);
+        }
+        else {
+            return { ...entry, cacheLayer: "memory" };
+        }
+    }
+    // Check L2 /tmp disk cache for 24-hour persistence across warm/cold workers
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `page_${sha256(key)}.json`);
+        if (fs.existsSync(diskPath)) {
+            const parsed = JSON.parse(fs.readFileSync(diskPath, "utf8"));
+            if (Date.now() <= parsed.expiresAt) {
+                pageCache.set(key, parsed);
+                return { ...parsed, cacheLayer: "disk" };
+            }
+            fs.unlinkSync(diskPath);
+        }
+    }
+    catch {
+        // ignore disk read errors
+    }
+    return undefined;
+}
+/**
+ * 3-tier read-through lookup: L1 Memory -> L2 /tmp Disk -> L3 Postgres `crawled_pages`.
+ * Promotes L3 Postgres hits back into L1 and L2 so repeat requests never re-crawl the target website.
+ */
+export async function getCachedPageAsync(url, retainImages) {
+    const localHit = getCachedPage(url, retainImages);
+    if (localHit) {
+        return localHit;
+    }
+    const { normalizedUrl } = normalizeCacheUrl(url);
+    const key = `${normalizedUrl}::${retainImages}`;
+    const urlHash = sha256(key);
+    try {
+        const db = await getOptionalDb();
+        const pgHit = await db?.getCrawledPage(urlHash);
+        if (pgHit) {
+            const cachedAt = new Date(pgHit.crawledAt).getTime();
+            const expiresAt = new Date(pgHit.expiresAt).getTime();
+            if (expiresAt > Date.now()) {
+                const promoted = {
+                    title: pgHit.title,
+                    markdown: pgHit.markdown,
+                    links: pgHit.links,
+                    structuredData: pgHit.structuredData ?? null,
+                    statusCode: pgHit.statusCode,
+                    contentHash: pgHit.contentHash,
+                    etag: pgHit.etag,
+                    lastModified: pgHit.lastModified,
+                    cacheLayer: "postgres",
+                    cachedAt,
+                    expiresAt,
+                };
+                pageCache.set(key, promoted);
+                try {
+                    const diskPath = path.join(TMP_CACHE_DIR, `page_${urlHash}.json`);
+                    fs.writeFileSync(diskPath, JSON.stringify(promoted));
+                }
+                catch {
+                    // ignore
+                }
+                return promoted;
+            }
+        }
+    }
+    catch {
+        // ignore DB lookup errors
+    }
+    return undefined;
+}
+export function setCachedPage(url, retainImages, entry, ttlMs = CACHE_TTL_MS) {
+    const { normalizedUrl, domain } = normalizeCacheUrl(url);
+    const key = `${normalizedUrl}::${retainImages}`;
+    const urlHash = sha256(key);
+    if (pageCache.size >= MAX_CACHE_ENTRIES) {
+        const oldestKey = pageCache.keys().next().value;
+        if (oldestKey) {
+            pageCache.delete(oldestKey);
+        }
+    }
+    const now = Date.now();
+    const fullEntry = {
+        ...entry,
+        cacheLayer: "memory",
+        cachedAt: now,
+        expiresAt: now + ttlMs,
+    };
+    pageCache.set(key, fullEntry);
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `page_${urlHash}.json`);
+        fs.writeFileSync(diskPath, JSON.stringify(fullEntry));
+    }
+    catch {
+        // ignore disk write errors
+    }
+    // Write-through to L3 Postgres `crawled_pages` store (when available)
+    void getOptionalDb()
+        .then((db) => db?.upsertCrawledPage({
+        urlHash,
+        normalizedUrl,
+        domain,
+        title: entry.title,
+        markdown: entry.markdown,
+        links: entry.links || [],
+        structuredData: entry.structuredData ?? null,
+        statusCode: entry.statusCode,
+        contentHash: entry.contentHash,
+        etag: entry.etag,
+        lastModified: entry.lastModified,
+        retainImages,
+        fetchCount: 1,
+        crawledAt: new Date(now).toISOString(),
+        expiresAt: new Date(now + ttlMs).toISOString(),
+    }))
+        .catch(() => { });
+}
+export function getCachedMap(cacheKey) {
+    const entry = mapCache.get(cacheKey);
+    if (entry) {
+        if (Date.now() > entry.expiresAt) {
+            mapCache.delete(cacheKey);
+        }
+        else {
+            return { ...entry, cacheLayer: "memory" };
+        }
+    }
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `map_${sha256(cacheKey)}.json`);
+        if (fs.existsSync(diskPath)) {
+            const parsed = JSON.parse(fs.readFileSync(diskPath, "utf8"));
+            if (Date.now() <= parsed.expiresAt) {
+                mapCache.set(cacheKey, parsed);
+                return { ...parsed, cacheLayer: "disk" };
+            }
+            fs.unlinkSync(diskPath);
+        }
+    }
+    catch {
+        // ignore
+    }
+    return undefined;
+}
+/**
+ * 3-tier read-through lookup for website maps/crawls: L1 Memory -> L2 Disk -> L3 Postgres `crawled_sites`.
+ */
+export async function getCachedMapAsync(cacheKey) {
+    const localHit = getCachedMap(cacheKey);
+    if (localHit) {
+        return localHit;
+    }
+    const cacheKeyHash = sha256(cacheKey);
+    try {
+        const db = await getOptionalDb();
+        const pgHit = await db?.getCrawledSite(cacheKeyHash);
+        if (pgHit) {
+            const cachedAt = new Date(pgHit.crawledAt).getTime();
+            const expiresAt = new Date(pgHit.expiresAt).getTime();
+            if (expiresAt > Date.now()) {
+                const promoted = {
+                    links: pgHit.links,
+                    source: pgHit.source,
+                    pagesCrawled: pgHit.pagesCrawled,
+                    cacheLayer: "postgres",
+                    cachedAt,
+                    expiresAt,
+                };
+                mapCache.set(cacheKey, promoted);
+                try {
+                    const diskPath = path.join(TMP_CACHE_DIR, `map_${cacheKeyHash}.json`);
+                    fs.writeFileSync(diskPath, JSON.stringify(promoted));
+                }
+                catch {
+                    // ignore
+                }
+                return promoted;
+            }
+        }
+    }
+    catch {
+        // ignore
+    }
+    return undefined;
+}
+export function setCachedMap(cacheKey, entry, ttlMs = CACHE_TTL_MS, rootUrl) {
+    const now = Date.now();
+    const cacheKeyHash = sha256(cacheKey);
+    const fullEntry = {
+        ...entry,
+        cacheLayer: "memory",
+        cachedAt: now,
+        expiresAt: now + ttlMs,
+    };
+    mapCache.set(cacheKey, fullEntry);
+    try {
+        const diskPath = path.join(TMP_CACHE_DIR, `map_${cacheKeyHash}.json`);
+        fs.writeFileSync(diskPath, JSON.stringify(fullEntry));
+    }
+    catch {
+        // ignore
+    }
+    const rawRoot = rootUrl || cacheKey.split("::")[0] || "";
+    const { normalizedUrl, domain } = normalizeCacheUrl(rawRoot);
+    void getOptionalDb()
+        .then((db) => db?.upsertCrawledSite({
+        cacheKeyHash,
+        rootUrl: normalizedUrl,
+        domain,
+        links: entry.links || [],
+        source: entry.source,
+        pagesCrawled: entry.pagesCrawled,
+        crawledAt: new Date(now).toISOString(),
+        expiresAt: new Date(now + ttlMs).toISOString(),
+    }))
+        .catch(() => { });
+}

package/dist/core/engine/chrome-cdp.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Minimal Chrome DevTools Protocol client for local JS rendering.
+ *
+ * Drive a real Chrome over CDP instead of dump-dom, wait until the DOM is
+ * actually ready, and always bound the session with a hard timeout. No
+ * puppeteer/playwright dependency.
+ */
+import { spawn } from "node:child_process";
+import { type ChromeHeadlessMode } from "./chrome-launch.js";
+export interface CdpSnapshot {
+    html: string;
+    title: string;
+    textLen: number;
+    readyState: string;
+}
+export interface CdpRenderOptions {
+    timeoutMs: number;
+    headless: ChromeHeadlessMode;
+    /** Injected for tests. */
+    spawnChrome?: typeof spawn;
+}
+export declare function renderUrlViaCdp(chromePath: string, url: string, opts: CdpRenderOptions): Promise<CdpSnapshot>;